package xyz.xiezc.spider.duplicate;

import cn.hutool.bloomfilter.BloomFilter;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import xyz.xiezc.ioc.annotation.Component;
import xyz.xiezc.ioc.annotation.Inject;
import xyz.xiezc.spider.common.Request;
import xyz.xiezc.spider.common.XBean;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * 分线程安全的去重链接, 主要去重的是下面需要爬去的链接
 *
 * @author wb-xzc291800
 * @date 2019/04/04 11:40
 */
@Slf4j
@Component
public class BloomFilterDuplicate implements Duplicate {
    /**
     * 布隆过滤器, 支持千万级数据过滤,
     * 10 大约相当于可以过滤1600w的数据
     */
    @Inject
    BloomFilter bloomFilter;


    @Synchronized
    @Override
    public void isRepeat(final XBean objectXBean) {
        List<Request> ret = new ArrayList<>();
        Request currentRequest = objectXBean.getRequest();
        log.info("Duplicate start; url:{}", currentRequest.getUrl());
        Map<String, Object> extra = currentRequest.getExtra();
        List<Request> nextRequests = objectXBean.getNextRequests();
        if (nextRequests == null || nextRequests.isEmpty()) {
            return;
        }
        for (Request nextReq : nextRequests) {
            if (StrUtil.isBlank(nextReq.getUrl())) {
                continue;
            }
            if (bloomFilter.add(nextReq.getUrl())) {
                nextReq.addExtra(extra);
                Map<String, List<String>> header = nextReq.getHeader();
                header.put("Referer", CollUtil.newArrayList(currentRequest.getUrl()));
                nextReq.setMethod("get");
                ret.add(nextReq);
            }
        }
        objectXBean.setNextRequests(ret);
        return;
    }

}

